# 夜魅小说网平台数据爬取
# 流浪小说平台数据爬取

import time
import re
import insert_sql
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import excel_spider_sql
list1  = excel_spider_sql.getConditions()
username = ['']
def zhangzhongyun_novel():
    for x in range(len(list1)):
        if list1[x]["Platform"] == "掌中云1":
            username = ""
            # Platform = "掌中云1"
        else:
            username = ""
            # Platform = "掌中云2"
        driver = webdriver.Chrome(r'D:\Program Files\python3.63\chromedriver.exe')
       
        url = "https://inovel.818tu.com/backend/referral_links/index"
        driver.get('{}'.format(url))
        driver.maximize_window()
        driver.implicitly_wait(5)
        driver.find_element_by_id('username').clear()
        driver.find_element_by_id('username').send_keys('{0}'.format(username))
        #定位密码
        time.sleep(0.5)
        driver.find_element_by_name('password').clear()
        driver.find_element_by_name('password').send_keys('{0}'.format(''))
        #点击登录
        driver.find_element_by_id('loginbtn').click()
        # 获取页面源码
        # body = driver.page_source
        # print(body)
        time.sleep(2)

        for m in range(4):
            try:
                time.sleep(2)
                # WebDriverWait(driver,15).until(
                #     EC.element_to_be_clickable((By.XPATH,"//button[text()='我知道了']"))
                # )
                driver.find_element_by_xpath("//button[text()='我知道了']").click()

            except Exception:
                print("未处理完弹出框问题")
                break
        #点击推广管理
        WebDriverWait(driver, 15).until(
            EC.element_to_be_clickable((By.XPATH, '//*[@id="menu-ad"]'))
        )
        driver.find_element_by_xpath('//*[@id="menu-ad"]').click()
        #点击推广链接显示推广页面
        driver.find_element_by_xpath('//*[@id="menu-ad-submenu"]/li[1]/a').click()
        #获取总页面
        # number = driver.find_element_by_xpath('//span[1]/em[2]').text
        # j = int(number) +1
        for y in range(1,5):
            # time.sleep(2)
            WebDriverWait(driver, 15).until(
                EC.element_to_be_clickable((By.XPATH, '//span[2]/input'))
            )
            driver.find_element_by_xpath('//span[2]/input').clear()
            driver.find_element_by_xpath('//span[2]/input').send_keys('{0}'.format(y))
            driver.find_element_by_xpath('//span[2]/button').click()
            body = driver.page_source
            # print(body)
            html = body.replace('\n','').replace('\r','').replace('\t','')
            # 公众平台
            Public_platform_list = re.findall('<span class="link-desc">(.*?)</span>',html)
            #推广链接url
            url_list = re.findall('<span class="text-primary" id=".*?">(.*?)</span>',html)
            #创建时间
            Creation_time_list = re.findall('创建时间:(.*?)</div>',html)
            # 推荐的小说名
            Novel_name_list = re.findall('<a href="/backend/novels/view/.*?">(.*?)</a>',html)
            #小说的章节
            # Novel_name_chapter_list = re.findall('<div style="line-height:1.7em">.*?</a>(.*?)</div>',html)
            #原文点击次数
            Number_clicks_list = re.findall('<span class="counter-field" data-field="clicks">(.*?)</span>',html)
            #新增阅读用户
            New_reading_users_list = re.findall('<span class="counter-field" data-field="member_count">(.*?)</span>',html)

            #新增关注
            New_attention_list = re.findall('<span class="counter-field" data-field="subscribe_count">(.*?)</span>',html)

            #充值金额
            Recharge_amount_list = re.findall('<span class="counter-field" data-field="subscribe_rate">.*?</td>.*?<td class="text-right">(.*?)</td>',html)
            #充值笔数
            Recharge_number_list = re.findall('<p><span class="counter-field">(.*?)</span> 笔</p>.*?<p><span class="counter-field">(.*?)</span> %</p>',html)
            # #充值比例
            # Recharge_proportion = re.findall('<p><span class="counter-field">(.*?)%</p>',html)
            for i in range(len(Public_platform_list)):
                Public_platform = Public_platform_list[i]
                url = url_list[i]
                Creation_time = Creation_time_list[i]
                Novel_name = Novel_name_list[i]
                Number_clicks = Number_clicks_list[i]
                New_reading_users = New_reading_users_list[i]
                New_attention = New_attention_list[i]
                Recharge_amount = Recharge_amount_list[i]
                Recharge_number = Recharge_number_list[i][0]
                # 判断数据库中是否已存在此推广链接
                if list1[x]["date_time"] in Public_platform:
                    if list1[x]["Platform"] == "掌中云1":
                        a = insert_sql.select_novel_mysql(url=url_list[i])
                        if a == 1:
                            # 存在 替换原有数据
                            insert_sql.Update_novel_mysql(Public_platform,url,Creation_time,Novel_name,Number_clicks,New_reading_users,New_attention,Recharge_amount,Recharge_number)
                            print("夜魅小说网 更新数据")
                        else:
                            # 否则  插入数据
                            insert_sql.insert_novel_mysql(Public_platform,url,Creation_time,Novel_name,Number_clicks,New_reading_users,New_attention,Recharge_amount,Recharge_number)
                            print("夜魅小说网 插入数据")
                    else:
                        a = insert_sql.liulang_select_novel_mysql(url=url_list[i])
                        if a == 1:
                            # 存在 替换原有数据
                            insert_sql.liulang_Update_novel_mysql(Public_platform,url,Creation_time,Novel_name,Number_clicks,New_reading_users,New_attention,Recharge_amount,Recharge_number)
                            print("流浪小说 更新数据")
                        else:
                            # 否则  插入数据
                            insert_sql.liulang_insert_novel_mysql(Public_platform,url,Creation_time,Novel_name,Number_clicks,New_reading_users,New_attention,Recharge_amount,Recharge_number)
                            print("流浪小说 插入数据")
                else:
                    pass
        driver.quit()
        #休眠
    print("休眠10分钟")
    time.sleep(600)
if __name__ == '__main__':
    zhangzhongyun_novel()